<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<!-- saved from url=(0014)about:internet -->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">

<title>Chater 6: Exercise 8</title>

<base target="_blank"/>

<style type="text/css">
body, td {
   font-family: sans-serif;
   background-color: white;
   font-size: 12px;
   margin: 8px;
}

tt, code, pre {
   font-family: 'DejaVu Sans Mono', 'Droid Sans Mono', 'Lucida Console', Consolas, Monaco, monospace;
}

h1 { 
   font-size:2.2em; 
}

h2 { 
   font-size:1.8em; 
}

h3 { 
   font-size:1.4em; 
}

h4 { 
   font-size:1.0em; 
}

h5 { 
   font-size:0.9em; 
}

h6 { 
   font-size:0.8em; 
}

a:visited {
   color: rgb(50%, 0%, 50%);
}

pre {	
   margin-top: 0;
   max-width: 95%;
   border: 1px solid #ccc;
}

pre code {
   display: block; padding: 0.5em;
}

code.r {
   background-color: #F8F8F8;
}

table, td, th {
  border: none;
}

blockquote {
   color:#666666;
   margin:0;
   padding-left: 1em;
   border-left: 0.5em #EEE solid;
}

hr {
   height: 0px;
   border-bottom: none;
   border-top-width: thin;
   border-top-style: dotted;
   border-top-color: #999999;
}

@media print {
   * { 
      background: transparent !important; 
      color: black !important; 
      filter:none !important; 
      -ms-filter: none !important; 
   }

   body { 
      font-size:12pt; 
      max-width:100%; 
   }
       
   a, a:visited { 
      text-decoration: underline; 
   }

   hr { 
      visibility: hidden;
      page-break-before: always;
   }

   pre, blockquote { 
      padding-right: 1em; 
      page-break-inside: avoid; 
   }

   tr, img { 
      page-break-inside: avoid; 
   }

   img { 
      max-width: 100% !important; 
   }

   @page :left { 
      margin: 15mm 20mm 15mm 10mm; 
   }
     
   @page :right { 
      margin: 15mm 10mm 15mm 20mm; 
   }

   p, h2, h3 { 
      orphans: 3; widows: 3; 
   }

   h2, h3 { 
      page-break-after: avoid; 
   }
}

</style>

<!-- Styles for R syntax highlighter -->
<style type="text/css">
   pre .operator,
   pre .paren {
     color: rgb(104, 118, 135)
   }

   pre .literal {
     color: rgb(88, 72, 246)
   }

   pre .number {
     color: rgb(0, 0, 205);
   }

   pre .comment {
     color: rgb(76, 136, 107);
   }

   pre .keyword {
     color: rgb(0, 0, 255);
   }

   pre .identifier {
     color: rgb(0, 0, 0);
   }

   pre .string {
     color: rgb(3, 106, 7);
   }
</style>

<!-- R syntax highlighter -->
<script type="text/javascript">
var hljs=new function(){function m(p){return p.replace(/&/gm,"&amp;").replace(/</gm,"&lt;")}function f(r,q,p){return RegExp(q,"m"+(r.cI?"i":"")+(p?"g":""))}function b(r){for(var p=0;p<r.childNodes.length;p++){var q=r.childNodes[p];if(q.nodeName=="CODE"){return q}if(!(q.nodeType==3&&q.nodeValue.match(/\s+/))){break}}}function h(t,s){var p="";for(var r=0;r<t.childNodes.length;r++){if(t.childNodes[r].nodeType==3){var q=t.childNodes[r].nodeValue;if(s){q=q.replace(/\n/g,"")}p+=q}else{if(t.childNodes[r].nodeName=="BR"){p+="\n"}else{p+=h(t.childNodes[r])}}}if(/MSIE [678]/.test(navigator.userAgent)){p=p.replace(/\r/g,"\n")}return p}function a(s){var r=s.className.split(/\s+/);r=r.concat(s.parentNode.className.split(/\s+/));for(var q=0;q<r.length;q++){var p=r[q].replace(/^language-/,"");if(e[p]){return p}}}function c(q){var p=[];(function(s,t){for(var r=0;r<s.childNodes.length;r++){if(s.childNodes[r].nodeType==3){t+=s.childNodes[r].nodeValue.length}else{if(s.childNodes[r].nodeName=="BR"){t+=1}else{if(s.childNodes[r].nodeType==1){p.push({event:"start",offset:t,node:s.childNodes[r]});t=arguments.callee(s.childNodes[r],t);p.push({event:"stop",offset:t,node:s.childNodes[r]})}}}}return t})(q,0);return p}function k(y,w,x){var q=0;var z="";var s=[];function u(){if(y.length&&w.length){if(y[0].offset!=w[0].offset){return(y[0].offset<w[0].offset)?y:w}else{return w[0].event=="start"?y:w}}else{return y.length?y:w}}function t(D){var A="<"+D.nodeName.toLowerCase();for(var B=0;B<D.attributes.length;B++){var C=D.attributes[B];A+=" "+C.nodeName.toLowerCase();if(C.value!==undefined&&C.value!==false&&C.value!==null){A+='="'+m(C.value)+'"'}}return A+">"}while(y.length||w.length){var v=u().splice(0,1)[0];z+=m(x.substr(q,v.offset-q));q=v.offset;if(v.event=="start"){z+=t(v.node);s.push(v.node)}else{if(v.event=="stop"){var p,r=s.length;do{r--;p=s[r];z+=("</"+p.nodeName.toLowerCase()+">")}while(p!=v.node);s.splice(r,1);while(r<s.length){z+=t(s[r]);r++}}}}return z+m(x.substr(q))}function j(){function q(x,y,v){if(x.compiled){return}var u;var s=[];if(x.k){x.lR=f(y,x.l||hljs.IR,true);for(var w in x.k){if(!x.k.hasOwnProperty(w)){continue}if(x.k[w] instanceof Object){u=x.k[w]}else{u=x.k;w="keyword"}for(var r in u){if(!u.hasOwnProperty(r)){continue}x.k[r]=[w,u[r]];s.push(r)}}}if(!v){if(x.bWK){x.b="\\b("+s.join("|")+")\\s"}x.bR=f(y,x.b?x.b:"\\B|\\b");if(!x.e&&!x.eW){x.e="\\B|\\b"}if(x.e){x.eR=f(y,x.e)}}if(x.i){x.iR=f(y,x.i)}if(x.r===undefined){x.r=1}if(!x.c){x.c=[]}x.compiled=true;for(var t=0;t<x.c.length;t++){if(x.c[t]=="self"){x.c[t]=x}q(x.c[t],y,false)}if(x.starts){q(x.starts,y,false)}}for(var p in e){if(!e.hasOwnProperty(p)){continue}q(e[p].dM,e[p],true)}}function d(B,C){if(!j.called){j();j.called=true}function q(r,M){for(var L=0;L<M.c.length;L++){if((M.c[L].bR.exec(r)||[null])[0]==r){return M.c[L]}}}function v(L,r){if(D[L].e&&D[L].eR.test(r)){return 1}if(D[L].eW){var M=v(L-1,r);return M?M+1:0}return 0}function w(r,L){return L.i&&L.iR.test(r)}function K(N,O){var M=[];for(var L=0;L<N.c.length;L++){M.push(N.c[L].b)}var r=D.length-1;do{if(D[r].e){M.push(D[r].e)}r--}while(D[r+1].eW);if(N.i){M.push(N.i)}return f(O,M.join("|"),true)}function p(M,L){var N=D[D.length-1];if(!N.t){N.t=K(N,E)}N.t.lastIndex=L;var r=N.t.exec(M);return r?[M.substr(L,r.index-L),r[0],false]:[M.substr(L),"",true]}function z(N,r){var L=E.cI?r[0].toLowerCase():r[0];var M=N.k[L];if(M&&M instanceof Array){return M}return false}function F(L,P){L=m(L);if(!P.k){return L}var r="";var O=0;P.lR.lastIndex=0;var M=P.lR.exec(L);while(M){r+=L.substr(O,M.index-O);var N=z(P,M);if(N){x+=N[1];r+='<span class="'+N[0]+'">'+M[0]+"</span>"}else{r+=M[0]}O=P.lR.lastIndex;M=P.lR.exec(L)}return r+L.substr(O,L.length-O)}function J(L,M){if(M.sL&&e[M.sL]){var r=d(M.sL,L);x+=r.keyword_count;return r.value}else{return F(L,M)}}function I(M,r){var L=M.cN?'<span class="'+M.cN+'">':"";if(M.rB){y+=L;M.buffer=""}else{if(M.eB){y+=m(r)+L;M.buffer=""}else{y+=L;M.buffer=r}}D.push(M);A+=M.r}function G(N,M,Q){var R=D[D.length-1];if(Q){y+=J(R.buffer+N,R);return false}var P=q(M,R);if(P){y+=J(R.buffer+N,R);I(P,M);return P.rB}var L=v(D.length-1,M);if(L){var O=R.cN?"</span>":"";if(R.rE){y+=J(R.buffer+N,R)+O}else{if(R.eE){y+=J(R.buffer+N,R)+O+m(M)}else{y+=J(R.buffer+N+M,R)+O}}while(L>1){O=D[D.length-2].cN?"</span>":"";y+=O;L--;D.length--}var r=D[D.length-1];D.length--;D[D.length-1].buffer="";if(r.starts){I(r.starts,"")}return R.rE}if(w(M,R)){throw"Illegal"}}var E=e[B];var D=[E.dM];var A=0;var x=0;var y="";try{var s,u=0;E.dM.buffer="";do{s=p(C,u);var t=G(s[0],s[1],s[2]);u+=s[0].length;if(!t){u+=s[1].length}}while(!s[2]);if(D.length>1){throw"Illegal"}return{r:A,keyword_count:x,value:y}}catch(H){if(H=="Illegal"){return{r:0,keyword_count:0,value:m(C)}}else{throw H}}}function g(t){var p={keyword_count:0,r:0,value:m(t)};var r=p;for(var q in e){if(!e.hasOwnProperty(q)){continue}var s=d(q,t);s.language=q;if(s.keyword_count+s.r>r.keyword_count+r.r){r=s}if(s.keyword_count+s.r>p.keyword_count+p.r){r=p;p=s}}if(r.language){p.second_best=r}return p}function i(r,q,p){if(q){r=r.replace(/^((<[^>]+>|\t)+)/gm,function(t,w,v,u){return w.replace(/\t/g,q)})}if(p){r=r.replace(/\n/g,"<br>")}return r}function n(t,w,r){var x=h(t,r);var v=a(t);var y,s;if(v){y=d(v,x)}else{return}var q=c(t);if(q.length){s=document.createElement("pre");s.innerHTML=y.value;y.value=k(q,c(s),x)}y.value=i(y.value,w,r);var u=t.className;if(!u.match("(\\s|^)(language-)?"+v+"(\\s|$)")){u=u?(u+" "+v):v}if(/MSIE [678]/.test(navigator.userAgent)&&t.tagName=="CODE"&&t.parentNode.tagName=="PRE"){s=t.parentNode;var p=document.createElement("div");p.innerHTML="<pre><code>"+y.value+"</code></pre>";t=p.firstChild.firstChild;p.firstChild.cN=s.cN;s.parentNode.replaceChild(p.firstChild,s)}else{t.innerHTML=y.value}t.className=u;t.result={language:v,kw:y.keyword_count,re:y.r};if(y.second_best){t.second_best={language:y.second_best.language,kw:y.second_best.keyword_count,re:y.second_best.r}}}function o(){if(o.called){return}o.called=true;var r=document.getElementsByTagName("pre");for(var p=0;p<r.length;p++){var q=b(r[p]);if(q){n(q,hljs.tabReplace)}}}function l(){if(window.addEventListener){window.addEventListener("DOMContentLoaded",o,false);window.addEventListener("load",o,false)}else{if(window.attachEvent){window.attachEvent("onload",o)}else{window.onload=o}}}var e={};this.LANGUAGES=e;this.highlight=d;this.highlightAuto=g;this.fixMarkup=i;this.highlightBlock=n;this.initHighlighting=o;this.initHighlightingOnLoad=l;this.IR="[a-zA-Z][a-zA-Z0-9_]*";this.UIR="[a-zA-Z_][a-zA-Z0-9_]*";this.NR="\\b\\d+(\\.\\d+)?";this.CNR="\\b(0[xX][a-fA-F0-9]+|(\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)";this.BNR="\\b(0b[01]+)";this.RSR="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|\\.|-|-=|/|/=|:|;|<|<<|<<=|<=|=|==|===|>|>=|>>|>>=|>>>|>>>=|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~";this.ER="(?![\\s\\S])";this.BE={b:"\\\\.",r:0};this.ASM={cN:"string",b:"'",e:"'",i:"\\n",c:[this.BE],r:0};this.QSM={cN:"string",b:'"',e:'"',i:"\\n",c:[this.BE],r:0};this.CLCM={cN:"comment",b:"//",e:"$"};this.CBLCLM={cN:"comment",b:"/\\*",e:"\\*/"};this.HCM={cN:"comment",b:"#",e:"$"};this.NM={cN:"number",b:this.NR,r:0};this.CNM={cN:"number",b:this.CNR,r:0};this.BNM={cN:"number",b:this.BNR,r:0};this.inherit=function(r,s){var p={};for(var q in r){p[q]=r[q]}if(s){for(var q in s){p[q]=s[q]}}return p}}();hljs.LANGUAGES.cpp=function(){var a={keyword:{"false":1,"int":1,"float":1,"while":1,"private":1,"char":1,"catch":1,"export":1,virtual:1,operator:2,sizeof:2,dynamic_cast:2,typedef:2,const_cast:2,"const":1,struct:1,"for":1,static_cast:2,union:1,namespace:1,unsigned:1,"long":1,"throw":1,"volatile":2,"static":1,"protected":1,bool:1,template:1,mutable:1,"if":1,"public":1,friend:2,"do":1,"return":1,"goto":1,auto:1,"void":2,"enum":1,"else":1,"break":1,"new":1,extern:1,using:1,"true":1,"class":1,asm:1,"case":1,typeid:1,"short":1,reinterpret_cast:2,"default":1,"double":1,register:1,explicit:1,signed:1,typename:1,"try":1,"this":1,"switch":1,"continue":1,wchar_t:1,inline:1,"delete":1,alignof:1,char16_t:1,char32_t:1,constexpr:1,decltype:1,noexcept:1,nullptr:1,static_assert:1,thread_local:1,restrict:1,_Bool:1,complex:1},built_in:{std:1,string:1,cin:1,cout:1,cerr:1,clog:1,stringstream:1,istringstream:1,ostringstream:1,auto_ptr:1,deque:1,list:1,queue:1,stack:1,vector:1,map:1,set:1,bitset:1,multiset:1,multimap:1,unordered_set:1,unordered_map:1,unordered_multiset:1,unordered_multimap:1,array:1,shared_ptr:1}};return{dM:{k:a,i:"</",c:[hljs.CLCM,hljs.CBLCLM,hljs.QSM,{cN:"string",b:"'\\\\?.",e:"'",i:"."},{cN:"number",b:"\\b(\\d+(\\.\\d*)?|\\.\\d+)(u|U|l|L|ul|UL|f|F)"},hljs.CNM,{cN:"preprocessor",b:"#",e:"$"},{cN:"stl_container",b:"\\b(deque|list|queue|stack|vector|map|set|bitset|multiset|multimap|unordered_map|unordered_set|unordered_multiset|unordered_multimap|array)\\s*<",e:">",k:a,r:10,c:["self"]}]}}}();hljs.LANGUAGES.r={dM:{c:[hljs.HCM,{cN:"number",b:"\\b0[xX][0-9a-fA-F]+[Li]?\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"number",b:"\\b\\d+(?:[eE][+\\-]?\\d*)?L\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"number",b:"\\b\\d+\\.(?!\\d)(?:i\\b)?",e:hljs.IMMEDIATE_RE,r:1},{cN:"number",b:"\\b\\d+(?:\\.\\d*)?(?:[eE][+\\-]?\\d*)?i?\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"number",b:"\\.\\d+(?:[eE][+\\-]?\\d*)?i?\\b",e:hljs.IMMEDIATE_RE,r:1},{cN:"keyword",b:"(?:tryCatch|library|setGeneric|setGroupGeneric)\\b",e:hljs.IMMEDIATE_RE,r:10},{cN:"keyword",b:"\\.\\.\\.",e:hljs.IMMEDIATE_RE,r:10},{cN:"keyword",b:"\\.\\.\\d+(?![\\w.])",e:hljs.IMMEDIATE_RE,r:10},{cN:"keyword",b:"\\b(?:function)",e:hljs.IMMEDIATE_RE,r:2},{cN:"keyword",b:"(?:if|in|break|next|repeat|else|for|return|switch|while|try|stop|warning|require|attach|detach|source|setMethod|setClass)\\b",e:hljs.IMMEDIATE_RE,r:1},{cN:"literal",b:"(?:NA|NA_integer_|NA_real_|NA_character_|NA_complex_)\\b",e:hljs.IMMEDIATE_RE,r:10},{cN:"literal",b:"(?:NULL|TRUE|FALSE|T|F|Inf|NaN)\\b",e:hljs.IMMEDIATE_RE,r:1},{cN:"identifier",b:"[a-zA-Z.][a-zA-Z0-9._]*\\b",e:hljs.IMMEDIATE_RE,r:0},{cN:"operator",b:"<\\-(?!\\s*\\d)",e:hljs.IMMEDIATE_RE,r:2},{cN:"operator",b:"\\->|<\\-",e:hljs.IMMEDIATE_RE,r:1},{cN:"operator",b:"%%|~",e:hljs.IMMEDIATE_RE},{cN:"operator",b:">=|<=|==|!=|\\|\\||&&|=|\\+|\\-|\\*|/|\\^|>|<|!|&|\\||\\$|:",e:hljs.IMMEDIATE_RE,r:0},{cN:"operator",b:"%",e:"%",i:"\\n",r:1},{cN:"identifier",b:"`",e:"`",r:0},{cN:"string",b:'"',e:'"',c:[hljs.BE],r:0},{cN:"string",b:"'",e:"'",c:[hljs.BE],r:0},{cN:"paren",b:"[[({\\])}]",e:hljs.IMMEDIATE_RE,r:0}]}};
hljs.initHighlightingOnLoad();
</script>


<!-- MathJax scripts -->
<script type="text/javascript" src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>



</head>

<body>
<h1>Chater 6: Exercise 8</h1>

<h3>a</h3>

<p>Create 100 \( X \) and \( \epsilon \) variables</p>

<pre><code class="r">set.seed(1)
X = rnorm(100)
eps = rnorm(100)
</code></pre>

<h3>b</h3>

<p>We are selecting \( \beta_0 = 3 \), \( \beta_1 = 2 \), \( \beta_2 = -3 \) and \( \beta_3 = 0.3 \).</p>

<pre><code class="r">beta0 = 3
beta1 = 2
beta2 = -3
beta3 = 0.3
Y = beta0 + beta1 * X + beta2 * X^2 + beta3 * X^3 + eps
</code></pre>

<h3>c</h3>

<p>Use \( regsubsets \) to select best model having polynomial of \( X \) of degree 10</p>

<pre><code class="r">library(leaps)
data.full = data.frame(y = Y, x = X)
mod.full = regsubsets(y ~ poly(x, 10, raw = T), data = data.full, nvmax = 10)
mod.summary = summary(mod.full)

# Find the model size for best cp, BIC and adjr2
which.min(mod.summary$cp)
</code></pre>

<pre><code>## [1] 3
</code></pre>

<pre><code class="r">which.min(mod.summary$bic)
</code></pre>

<pre><code>## [1] 3
</code></pre>

<pre><code class="r">which.max(mod.summary$adjr2)
</code></pre>

<pre><code>## [1] 3
</code></pre>

<pre><code class="r"># Plot cp, BIC and adjr2
plot(mod.summary$cp, xlab = &quot;Subset Size&quot;, ylab = &quot;Cp&quot;, pch = 20, type = &quot;l&quot;)
points(3, mod.summary$cp[3], pch = 4, col = &quot;red&quot;, lwd = 7)
</code></pre>

<p><img src="" alt="plot of chunk 8c"/> </p>

<pre><code class="r">plot(mod.summary$bic, xlab = &quot;Subset Size&quot;, ylab = &quot;BIC&quot;, pch = 20, type = &quot;l&quot;)
points(3, mod.summary$bic[3], pch = 4, col = &quot;red&quot;, lwd = 7)
</code></pre>

<p><img src="" alt="plot of chunk 8c"/> </p>

<pre><code class="r">plot(mod.summary$adjr2, xlab = &quot;Subset Size&quot;, ylab = &quot;Adjusted R2&quot;, pch = 20, 
    type = &quot;l&quot;)
points(3, mod.summary$adjr2[3], pch = 4, col = &quot;red&quot;, lwd = 7)
</code></pre>

<p><img src="" alt="plot of chunk 8c"/> </p>

<p>We find that with Cp, BIC and Adjusted R2 criteria, \( 3 \), \( 3 \), and \( 3 \) variable models are respectively picked. </p>

<pre><code class="r">coefficients(mod.full, id = 3)
</code></pre>

<pre><code>##           (Intercept) poly(x, 10, raw = T)1 poly(x, 10, raw = T)2 
##               3.07627               2.35624              -3.16515 
## poly(x, 10, raw = T)7 
##               0.01047
</code></pre>

<p>All statistics pick \( X^7 \) over \( X^3 \). The remaining coefficients are quite close to \( \beta \) s.</p>

<h3>d</h3>

<p>We fit forward and backward stepwise models to the data.</p>

<pre><code class="r">mod.fwd = regsubsets(y ~ poly(x, 10, raw = T), data = data.full, nvmax = 10, 
    method = &quot;forward&quot;)
mod.bwd = regsubsets(y ~ poly(x, 10, raw = T), data = data.full, nvmax = 10, 
    method = &quot;backward&quot;)
fwd.summary = summary(mod.fwd)
bwd.summary = summary(mod.bwd)
which.min(fwd.summary$cp)
</code></pre>

<pre><code>## [1] 3
</code></pre>

<pre><code class="r">which.min(bwd.summary$cp)
</code></pre>

<pre><code>## [1] 3
</code></pre>

<pre><code class="r">which.min(fwd.summary$bic)
</code></pre>

<pre><code>## [1] 3
</code></pre>

<pre><code class="r">which.min(bwd.summary$bic)
</code></pre>

<pre><code>## [1] 3
</code></pre>

<pre><code class="r">which.max(fwd.summary$adjr2)
</code></pre>

<pre><code>## [1] 3
</code></pre>

<pre><code class="r">which.max(bwd.summary$adjr2)
</code></pre>

<pre><code>## [1] 4
</code></pre>

<pre><code class="r"># Plot the statistics
par(mfrow = c(3, 2))
plot(fwd.summary$cp, xlab = &quot;Subset Size&quot;, ylab = &quot;Forward Cp&quot;, pch = 20, type = &quot;l&quot;)
points(3, fwd.summary$cp[3], pch = 4, col = &quot;red&quot;, lwd = 7)
plot(bwd.summary$cp, xlab = &quot;Subset Size&quot;, ylab = &quot;Backward Cp&quot;, pch = 20, type = &quot;l&quot;)
points(3, bwd.summary$cp[3], pch = 4, col = &quot;red&quot;, lwd = 7)
plot(fwd.summary$bic, xlab = &quot;Subset Size&quot;, ylab = &quot;Forward BIC&quot;, pch = 20, 
    type = &quot;l&quot;)
points(3, fwd.summary$bic[3], pch = 4, col = &quot;red&quot;, lwd = 7)
plot(bwd.summary$bic, xlab = &quot;Subset Size&quot;, ylab = &quot;Backward BIC&quot;, pch = 20, 
    type = &quot;l&quot;)
points(3, bwd.summary$bic[3], pch = 4, col = &quot;red&quot;, lwd = 7)
plot(fwd.summary$adjr2, xlab = &quot;Subset Size&quot;, ylab = &quot;Forward Adjusted R2&quot;, 
    pch = 20, type = &quot;l&quot;)
points(3, fwd.summary$adjr2[3], pch = 4, col = &quot;red&quot;, lwd = 7)
plot(bwd.summary$adjr2, xlab = &quot;Subset Size&quot;, ylab = &quot;Backward Adjusted R2&quot;, 
    pch = 20, type = &quot;l&quot;)
points(4, bwd.summary$adjr2[4], pch = 4, col = &quot;red&quot;, lwd = 7)
</code></pre>

<p><img src="" alt="plot of chunk 8d"/> </p>

<p>We see that all statistics pick \( 3 \) variable models except backward stepwise with adjusted R2. Here are the coefficients</p>

<pre><code class="r">coefficients(mod.fwd, id = 3)
</code></pre>

<pre><code>##           (Intercept) poly(x, 10, raw = T)1 poly(x, 10, raw = T)2 
##               3.07627               2.35624              -3.16515 
## poly(x, 10, raw = T)7 
##               0.01047
</code></pre>

<pre><code class="r">coefficients(mod.bwd, id = 3)
</code></pre>

<pre><code>##           (Intercept) poly(x, 10, raw = T)1 poly(x, 10, raw = T)2 
##               3.07888               2.41982              -3.17724 
## poly(x, 10, raw = T)9 
##               0.00187
</code></pre>

<pre><code class="r">coefficients(mod.fwd, id = 4)
</code></pre>

<pre><code>##           (Intercept) poly(x, 10, raw = T)1 poly(x, 10, raw = T)2 
##              3.112359              2.369859             -3.275727 
## poly(x, 10, raw = T)4 poly(x, 10, raw = T)7 
##              0.027674              0.009997
</code></pre>

<p>Here forward stepwise picks \( X^7 \) over \( X^3 \). Backward stepwise with \( 3 \) variables picks \( X^9 \) while backward stepwise with \( 4 \) variables picks \( X^4 \) and \( X^7 \). All other coefficients are close to \( \beta \) s.</p>

<h3>e</h3>

<p>Training Lasso on the data</p>

<pre><code class="r">library(glmnet)
</code></pre>

<pre><code>## Loading required package: Matrix
## Loading required package: lattice
## Loaded glmnet 1.9-5
</code></pre>

<pre><code class="r">xmat = model.matrix(y ~ poly(x, 10, raw = T), data = data.full)[, -1]
mod.lasso = cv.glmnet(xmat, Y, alpha = 1)
best.lambda = mod.lasso$lambda.min
best.lambda
</code></pre>

<pre><code>## [1] 0.03991
</code></pre>

<pre><code class="r">plot(mod.lasso)
</code></pre>

<p><img src="" alt="plot of chunk 8e"/> </p>

<pre><code class="r"># Next fit the model on entire data using best lambda
best.model = glmnet(xmat, Y, alpha = 1)
predict(best.model, s = best.lambda, type = &quot;coefficients&quot;)
</code></pre>

<pre><code>## 11 x 1 sparse Matrix of class &quot;dgCMatrix&quot;
##                                 1
## (Intercept)             3.0398151
## poly(x, 10, raw = T)1   2.2303371
## poly(x, 10, raw = T)2  -3.1033193
## poly(x, 10, raw = T)3   .        
## poly(x, 10, raw = T)4   .        
## poly(x, 10, raw = T)5   0.0498411
## poly(x, 10, raw = T)6   .        
## poly(x, 10, raw = T)7   0.0008068
## poly(x, 10, raw = T)8   .        
## poly(x, 10, raw = T)9   .        
## poly(x, 10, raw = T)10  .
</code></pre>

<p>Lasso also picks \( X^5 \) over \( X^3 \). It also picks \( X^7 \) with negligible coefficient.</p>

<h3>f</h3>

<p>Create new Y with different \( \beta_7 = 7 \).</p>

<pre><code class="r">beta7 = 7
Y = beta0 + beta7 * X^7 + eps
# Predict using regsubsets
data.full = data.frame(y = Y, x = X)
mod.full = regsubsets(y ~ poly(x, 10, raw = T), data = data.full, nvmax = 10)
mod.summary = summary(mod.full)

# Find the model size for best cp, BIC and adjr2
which.min(mod.summary$cp)
</code></pre>

<pre><code>## [1] 2
</code></pre>

<pre><code class="r">which.min(mod.summary$bic)
</code></pre>

<pre><code>## [1] 1
</code></pre>

<pre><code class="r">which.max(mod.summary$adjr2)
</code></pre>

<pre><code>## [1] 4
</code></pre>

<pre><code class="r">coefficients(mod.full, id = 1)
</code></pre>

<pre><code>##           (Intercept) poly(x, 10, raw = T)7 
##                 2.959                 7.001
</code></pre>

<pre><code class="r">coefficients(mod.full, id = 2)
</code></pre>

<pre><code>##           (Intercept) poly(x, 10, raw = T)2 poly(x, 10, raw = T)7 
##                3.0705               -0.1417                7.0016
</code></pre>

<pre><code class="r">coefficients(mod.full, id = 4)
</code></pre>

<pre><code>##           (Intercept) poly(x, 10, raw = T)1 poly(x, 10, raw = T)2 
##                3.0763                0.2914               -0.1618 
## poly(x, 10, raw = T)3 poly(x, 10, raw = T)7 
##               -0.2527                7.0091
</code></pre>

<p>We see that BIC picks the most accurate 1-variable model with matching coefficients. Other criteria pick additional variables.</p>

<pre><code class="r">xmat = model.matrix(y ~ poly(x, 10, raw = T), data = data.full)[, -1]
mod.lasso = cv.glmnet(xmat, Y, alpha = 1)
best.lambda = mod.lasso$lambda.min
best.lambda
</code></pre>

<pre><code>## [1] 12.37
</code></pre>

<pre><code class="r">best.model = glmnet(xmat, Y, alpha = 1)
predict(best.model, s = best.lambda, type = &quot;coefficients&quot;)
</code></pre>

<pre><code>## 11 x 1 sparse Matrix of class &quot;dgCMatrix&quot;
##                            1
## (Intercept)            3.820
## poly(x, 10, raw = T)1  .    
## poly(x, 10, raw = T)2  .    
## poly(x, 10, raw = T)3  .    
## poly(x, 10, raw = T)4  .    
## poly(x, 10, raw = T)5  .    
## poly(x, 10, raw = T)6  .    
## poly(x, 10, raw = T)7  6.797
## poly(x, 10, raw = T)8  .    
## poly(x, 10, raw = T)9  .    
## poly(x, 10, raw = T)10 .
</code></pre>

<p>Lasso also picks the best 1-variable model but intercet is quite off (\( 3.8 \) vs \( 3 \)).</p>

</body>

</html>

